package com.asiainfo.zqx;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import org.json.JSONArray;
import org.json.JSONObject;
import org.json.JSONException;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class jinrongjiewang extends BreadthCrawler {
    private String seedurl = "http://finance.jrj.com.cn/";
   // private String acturl = "http://stock.jrj.com.cn/share/news/articlepart/article.js";

    public jinrongjiewang(String crawlPath) {
        super(crawlPath, false);
        addSeed(seedurl);
  //      addSeed(acturl);
        setThreads(1);
        // setResumable(true);
    }

    @Override
    public void visit(Page page, CrawlDatums crawlDatums) {
        String contentType = page.contentType();
        if (contentType == null) {
            return;
        }
        if (page.matchType("commen")
                && page.select("div.titmain>p.inftop>span>i").first().text().contains("金融界")) {
            System.out.println("连接为" + page.url());
            System.out.println("标题为" + page.select("div.titmain>h1>p").text());
            System.out.println("时间为" + page.select("div.titmain>p.inftop>span").first().text().substring(0, 20));
            /*选择作者，先判断<em>是否为空,不为空，选择里面的内容作为作者，
            为空，选择<i>里面的内容作为作者，不能以超链接的为本作为根据，可能不是超链接
             */
            String writer;
            String emstr = page.select("div.titmain>p.inftop>span>i>em").text();
            String istr = page.select("div.titmain>p.inftop>span>i").text();
            if (emstr.length() > 3) {
                writer = emstr.substring(3);
            } else {
                writer = istr.substring(3);
                System.out.println("来源为" + writer);
            }
            System.out.println("作者为" + writer);
            System.out.println("正文为" + page.select("div.texttit_m1").text());
        } else if (page.matchType("bc") && page.select("div.titmain>p.inftop>span").get(1).text().contains("金融界")) {
            System.out.println("连接为" + page.url());
            System.out.println("标题为" + page.select("div.titmain>h1>p").text());
            System.out.println("时间为" + page.select("div.titmain>p.inftop>span").first().text());
            /*选择作者，先判断<em>是否为空,不为空，选择里面的内容作为作者，
            为空，选择<i>里面的内容作为作者，不能以超链接的为本作为根据，可能不是超链接
             */
            String writer;
            String istr = page.select("div.titmain>p.inftop>span").get(1).text();
            String emstr = page.select("div.titmain>p.inftop>span").get(2).text();

            if (emstr.length() > 3) {
                writer = emstr.substring(3);
            } else {
                writer = istr.substring(3);
                System.out.println("来源为" + writer);
            }
            System.out.println("作者为" + writer);
            System.out.println("正文为" + page.select("div.texttit_m1").text());
        } else if (page.matchType("fund") && page.select("div.mh-title>span.time>i>a").text().contains("金融界")) {
            System.out.println("连接为" + page.url());
            System.out.println("标题为" + page.select("div.texttitbox>h1>p").text());
            System.out.println("时间为" + page.select("div.mh-title>span.time").first().text().substring(0, 20));
            /*选择作者，先判断<em>是否为空,不为空，选择里面的内容作为作者，
            为空，选择<i>里面的内容作为作者，不能以超链接的为本作为根据，可能不是超链接
             */
            String writer;
            String istr = page.select("div.mh-title>span.time>i>em").text();
            String emstr = page.select("div.titmain>p.inftop>span").text();

            if (emstr.length() > 3) {
                writer = emstr.substring(3);
            } else {
                writer = istr.substring(3);
                System.out.println("来源为" + writer);
            }
            System.out.println("作者为" + writer);
            System.out.println("正文为" + page.select("div.texttit_m1").text());
        } else if(!page.matchType("bc")&&!page.matchType("fund")&&!page.matchType("commen")){
            Elements elements = page.select("a");
            System.out.println("共"+elements.size()+"中结果");
            for (Element element : elements) {
                String url = element.attr("abs:href");

                //分类处理，fina,biz,bc三种网页
                // url.contains("/2022/")&&url.contains(".shtml"))
                if (url.matches(".*/2022/.*.shtml") ){
                    if (url.contains("bc.jrj.com.cn/")) {
                        crawlDatums.add(new CrawlDatum(url, "bc"));
                        System.out.println("bc类型"+url);
                    }  else if (url.contains("fund.jrj.com.cn/")) {
                        crawlDatums.add(new CrawlDatum(url, "fund"));
                        System.out.println("fund类型"+url);
                    }else {
                        crawlDatums.add(new CrawlDatum(url, "commen"));
                        System.out.println("commen类型"+url);
                    }

                }
            }
        }
       /* else if (page.equals("http://stock.jrj.com.cn/share/news/articlepart/article.js")) {
            String text = page.html().toString();
            text = text.substring(text.indexOf("["), text.length() - 2);

            JSONArray jstor = null;
            try {
                jstor = new JSONArray(text);
                for (int i = 0; i < jstor.length(); i++) {
                    JSONObject jsonObject = (JSONObject) jstor.get(i);
                    String url = (String) jsonObject.get("infourl");
                    System.out.println("获得的url为" + url);
                    if (url.contains("/2022/")) {
                        if (url.contains("bc.jrj.com.cn/")) {
                            crawlDatums.add(new CrawlDatum(url, "bc"));
                        } else if (url.contains("fund.jrj.com.cn/")) {
                            crawlDatums.add(new CrawlDatum(url, "fund"));
                        } else {
                            crawlDatums.add(new CrawlDatum(url, "commen"));
                        }
                    }
                }
            } catch (JSONException e) {
                e.printStackTrace();
            }
        }*/
    }

    public static void main(String[] args) throws Exception {
        jinrongjiewang jin = new jinrongjiewang("jinrong");
        jin.start(2);
    }
}
